package cn.buaa.HLJ.Crawler;

import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpHost;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;

import cn.buaa.HLJ.Entity.ProjectUrl;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class LawCrawler  implements  PageProcessor,Runnable{

	private int  pageid=1;
	private int  pagemax=92;
	private Site site;
	private SqlSession session;
	private String moban="http://s.wanfangdata.com.cn/Cstad.aspx?q=CLCShort%3a%22D9*%22&f=subjectCatagory&p=";
	
	
	public LawCrawler() throws IOException
	{
		String resource="mybatis_config.xml";
		Reader reader=Resources.getResourceAsReader(resource);
		SqlSessionFactory sessionFactory=new SqlSessionFactoryBuilder().build(reader);
		session=sessionFactory.openSession();	
	}
	
	@Override
	public void run() {
		String s=moban+pageid;
		List<String> url = new ArrayList<String>();
		url.clear();
		url.add(s.replaceAll("\"", "%22"));
		Spider.create(this).startUrls(url).run(); 
	}

	@Override
	public Site getSite() {
		//site = Site.me().setSleepTime(1000).setRetryTimes(5).setCycleRetryTimes(3).setTimeOut(60000).setHttpProxy(new HttpHost("202.194.101.150",80));
		site = Site.me().setSleepTime(1000).setRetryTimes(5)
				.setCycleRetryTimes(3).setTimeOut(60000)
				//.setHttpProxy(new HttpHost("122.70.178.242",8118))
				;
		site.setCharset("UTF-8");
		return site;
	}

	@Override
	public void process(Page page) {
		List<String> links=page.getHtml().xpath("//div[@class='record-item-list']/div[@class='record-item']//div[@class='record-title']/a/@href").all();
		for(String one:links)
		{
			System.out.println(one);
		}
		
		List<String> titles=page.getHtml().xpath("//div[@class='record-item-list']/div[@class='record-item']//div[@class='record-title']/a/text()").all();
		for(String one:titles)
		{
			System.out.println(one);
		}
		
		//
		String statement="cn.buaa.HLJ.Mapper.ProjectUrlMapper.insertProjectUrl";
		
		for(int i=0;i<links.size();i++)
		{
			ProjectUrl url=new ProjectUrl();
			url.setTitle(titles.get(i));
			url.setUrl(links.get(i));
			
			session.insert(statement, url);
		}
		session.commit();
		
		if(pageid<=pagemax)
		{
			pageid++;
			String nextpage=moban+pageid;
			page.addTargetRequest(nextpage);
		}
	}
	
	public static void main(String[] args) throws IOException{
		new LawCrawler().run();
		System.out.println("pass");
	}

}
